#!/usr/bin/env python
# encoding: utf-8
"""
FullTextSearch.py

Created by Rui Carmo on 2007-02-19.
Published under the MIT license.
"""

from BeautifulSoup import *
from Engine import renderPage
from Utils import *
import os, time, gc
import threading, urlparse
from lupy.index.indexwriter import IndexWriter
from lupy import document
from lupy.index.term import Term
from lupy.search.indexsearcher import IndexSearcher
from lupy.search.term import TermQuery
from lupy.search.phrase import PhraseQuery
from lupy.search.boolean import BooleanQuery

class Indexer(threading.Thread):
  def __init__(self, appcontext, path, store, cache, staging):
    threading.Thread.__init__(self)
    self.path = path
    self.staging = staging
    c = self.appcontext = appcontext
    self.ready = False
    self.allpages = {}
    self.done = False
    try: # to restore previously indexed data upon startup
      self.pageinfo      = c.persistent['indexer:pageinfo']
      self.backlinks     = c.persistent['indexer:backlinks']
      self.wikilinks     = c.persistent['indexer:wikilinks']
      self.defaultlinks  = c.persistent['indexer:defaultlinks']
      self.wantedlinks   = c.persistent['indexer:wantedlinks']
      self.indexed       = c.persistent['indexer:indexed']
      # build the 'recent' helper array
      self.recent  = [name for name in self.pageinfo.keys()]
      self.recent.sort(lambda x, y: cmp(self.pageinfo[y]['last-modified'],self.pageinfo[x]['last-modified']))
      self.done = True
    except: # go at it from scratch
      self.pageinfo      = {} # all page headers
      self.backlinks     = {} # all backlinks across pages
      self.wikilinks     = {} # all wikilinks across pages
      self.defaultlinks  = {} # default links associated with specific pages
      self.wantedlinks   = [] # all missing pages
      self.recent = []      # recently modified pages
      self.indexed = {}  # page name and time of last indexing
    self.modified = [] # all pages, sorted by on-disk modification date
    self.aliases = {}  # page aliases
    self.working = True
  
  def stop(self):
    self.working = False
  
  def run(self):
    self.pagescan() # Do preliminary scanning
    gc.collect() # make sure we release memory
    # Wait for 30s before starting full-text indexing
    for i in range(0,6):
      time.sleep(5)
      if not self.working:
        return
    # Play nice and allow for thread to be killed externally with minimum delay
    while(self.fullscan()):
      for i in range(0,360):
        time.sleep(5)
        if not self.working:
          return
  
  def fullscan(self):
    """Full text index of content"""
    if self.staging:
      return
    print "Starting indexing..."
    self.pagescan()
    print "Rebuilding..."
    c = self.appcontext
    # Check if a previous fulltext index already exists
    if(not os.path.exists(self.path)):
      os.makedirs(self.path)
      self.writer = IndexWriter(self.path, True)
      self.indexed = {}
    # Use a previously created index
    else:
      self.writer = IndexWriter(self.path, False) 
    
    dirty = False
    count = 1
    bound = len(self.modified)
    # Go through all pages, starting with most recently modified files on disk
    for name in self.modified:
      if self.working == False:
        self.writer.close()
        return False
      
      if name not in self.backlinks:
        self.backlinks[name] = []
      if name not in self.wikilinks:
        self.wikilinks[name] = []
      
      try:
        # was the page modified on disk since we last saw it?
        if c.store.mtime(name) != self.indexed[name]:
          raise KeyError
        else:
          headers = self.pageinfo[name]
          if not self.done:
            time.sleep(0.2)
          continue
      except KeyError:
        # Get the page
        start = time.time()
        try:
          page = c.store.getRevision(name)
        except:
          print u'Error: Could not index %s, will be re-visited upon next indexing pass.' % name
          count = count + 1
          continue
        
        # Cache the header info
        self.pageinfo[name] = headers = page.headers
        # If pages have an "x-link" header then add it
        if "x-link" in headers:
          self.defaultlinks[name] = page.headers['x-link']
        
        # If pages are non-indexable just flag them as indexed and continue the loop
        if "x-index" in headers:
          if page.headers["x-index"].lower() == "no":
            self.indexed[name] = c.store.mtime(name)
            count = count + 1
            continue

        wikilinks = []
        # Start building inter-page links by resolving tags
        parse = ['keywords','categories','tags'] # headers to parse
        for header in parse:
          if header in page.headers:
            tags = [tag.strip().lower() for tag in headers[header].split(',')]
            for tag in tags:
              if tag in self.aliases.keys():
                wikilinks.append(self.aliases[tag])
        
        # Grab the page HTML
        soup = BeautifulSoup(renderPage(self.appcontext,page,cache = False))

        # Identify and store all wikilinks in markup
        links = soup.findAll('a',{'class':'wiki'})
        for link in links:
          (schema,netloc,path,parameters,query,fragment) = urlparse.urlparse(link['href'])
          path = path[len(self.appcontext.base):]
          path = self.resolveAlias(path)
          if not path in wikilinks:
            wikilinks.append(path)
        self.wikilinks[name] = wikilinks
          
        # Identify and store all unknown wiki links
        links = soup.findAll('a',{'class':'wikiunknown'})
        for link in links:
          (schema,netloc,path,parameters,query,fragment) = urlparse.urlparse(link['href'])
          path = path[len(self.appcontext.base):]
          if path.lower() in self.aliases.keys():
            path = self.aliases[path.lower()]
          if not path in self.wantedlinks:
            self.wantedlinks.append(path)
        
        # Gather together all plaintext
        plaintext = u' '.join(soup.findAll(text=re.compile('.+')))
        
        # TODO: add explicit tags, categories, the works
                
        # Store plaintext for displaying search results and search engine indexing
        self.appcontext.cache['plaintext:'+name] = plaintext
        
        # Perform backlinks update across pages
        for href in self.wikilinks[name]:
          if href not in self.backlinks:
            self.backlinks[href] = []
          if name not in self.backlinks[href]:
            self.backlinks[href].append(name)
        
        # Index the plaintext using lupy
        self.index(soup, page.headers, name, plaintext, self.wikilinks[name])
        dirty = True
        # Store a checkpoint
        c.persistent['indexer:indexed'] = self.indexed
        print "Info: %s done (%d of %d): %fs" % (name, count, bound, time.time()-start)
        count = count + 1
        self.indexed[name] = c.store.mtime(name)
    if dirty:
      print "Optimizing Index..."
      self.writer.optimize()
      print "Closing Index..."
      self.writer.close()
      print "Freeing RAM..."
      gc.collect()
      print "Saving context snapshot..."
      c.persistent['indexer:backlinks'] = self.backlinks
      c.persistent['indexer:wikilinks'] = self.wikilinks    
      c.persistent['indexer:defaultlinks'] = self.defaultlinks
      c.persistent['indexer:pageinfo'] = self.pageinfo
      c.persistent['indexer:wantedlinks'] = self.wantedlinks
      c.persistent['indexer:indexed'] = self.indexed
      self.recent  = [name for name in self.pageinfo.keys()]
      self.recent.sort(lambda x, y: cmp(self.pageinfo[y]['last-modified'],self.pageinfo[x]['last-modified']))
      print "Indexing complete."
      # if we have pinging enabled, then do it now
      if 'ping' in c.siteinfo:
        doPings(c.siteinfo)
    else:
      print "No index updates performed."
    self.done = True
    return True
        
  def pagescan(self):
    """Perform a shallow page scan"""
    c = self.appcontext
    self.allpages = c.store.allPages()
    self.aliases = c.store.aliases
    self.modified = self.allpages.keys()
    self.modified.sort(lambda x, y: self.allpages[y]-self.allpages[x])
    self.ready = True    
  
  def index(self, soup, headers, name, plaintext, wikilinks):
    """Index a single page"""
    d = document.Document()
    k = document.Keyword('name',name)
    d.add(k)
    if 'title' in headers.keys():
      titletext = headers['title']
    else:
      try:
        # search for initial header (if any)
        title = soup.findAll('h1')[0]
        titletext = u''.join(title.findAll(text=re.compile('.+')))
      except:
        pass
    # Add title
    k = document.Keyword('title',titletext)
    d.add(k)
    if not self.done:
      time.sleep(0.2) # play nice

    # now add all wikilinks as keywords
    # TODO: add real keywords from headers
    for link in wikilinks:
      k = document.Keyword('wikilink',link)
      d.add(k)
    t = document.Text('text', plaintext + " " + titletext, False)
    d.add(t)
    self.writer.addDocument(d)

  # TODO: http://tinyurl.com/23esyc
  # TODO: http://tinyurl.com/3xkza9
  
  def search(self, query):
    """Perform a query"""
    if not self.done:
      return None
    s = IndexSearcher(self.path)
    # Split query arguments
    query = query.split(' ')
    q = BooleanQuery()
    # Process arguments as a Boolean "and" query
    for a in query:
      t = Term('text', a)
      tq = TermQuery(t)
      q.add(tq, True, False)
      
    # Perform query
    hits = s.search(q)
    result = {}
    if len(hits) != 0:
      # Assemble result hash
      # TODO: add scoring
      for hit in hits:
        result[hit.get('name')] = { 'title': hit.get('title'), 'last-modified': self.pageinfo[hit.get('name')]['last-modified']}
    s.close()
    return result 
  
  def resolveAlias(self,path):
    """Resolve page aliases - placed here to allow for future re-use of indexing data"""
    for replacement in ALIASING_CHARS:
      alias = path.lower().replace(' ',replacement)
      if alias in self.aliases.keys():
        return self.aliases[alias]
    return path
